Amazon Bedrock のカスタムモデルインポート機能を試してみる

# Make sure you have git-lfs installed (https://git-lfs.com)
git lfs install

# モデルのダウンロード
git clone https://huggingface.co/karakuri-ai/karakuri-lm-8x7b-chat-v0.1

{
	"add_bos_token": true,
	"add_eos_token": false,
	"added_tokens_decoder": {
		"0": {
			"content": "<unk>",
			"lstrip": false,
			"normalized": false,
			"rstrip": false,
			"single_word": false,
			"special": true
		},
		"1": {
			"content": "<s>",
			"lstrip": false,
			"normalized": false,
			"rstrip": false,
			"single_word": false,
			"special": true
		},
		"2": {
			"content": "</s>",
			"lstrip": false,
			"normalized": false,
			"rstrip": false,
			"single_word": false,
			"special": true
		}
	},
	"additional_special_tokens": [],
	"bos_token": "<s>",
+	"chat_template": "{%- if messages[0]['role'] == 'system' %}\n    {%- set system_message = messages[0]['content'] %}\n    {%- set loop_messages = messages[1:] %}\n{%- else %}\n    {%- set loop_messages = messages %}\n{%- endif %}\n\n{{- bos_token }}\n{%- for message in loop_messages %}\n    {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}\n        {{- raise_exception('After the optional system message, conversation roles must alternate user/assistant/user/assistant/...') }}\n    {%- endif %}\n    {%- if message['role'] == 'user' %}\n        {%- if loop.first and system_message is defined %}\n            {{- ' [INST] ' + system_message + '\\n\\n' + message['content'] + ' [/INST]' }}\n        {%- else %}\n            {{- ' [INST] ' + message['content'] + ' [/INST]' }}\n        {%- endif %}\n    {%- elif message['role'] == 'assistant' %}\n        {{- ' ' + message['content'] + eos_token}}\n    {%- else %}\n        {{- raise_exception('Only user and assistant roles are supported, with the exception of an initial optional system message!') }}\n    {%- endif %}\n{%- endfor %}\n",
-       "chat_template": "{{ bos_token }}{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% for message in loop_messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if loop.index0 == 0 and system_message != false %}{% set content = '<<SYS>>\\n' + system_message + '\\n<</SYS>>\\n\\n' + message['content'] %}{% else %}{% set content = message['content'] %}{% endif %}{% if message['role'] == 'user' %}{% set helpfulness = message['helpfulness']|string or '4' %}{% set correctness = message['correctness']|string or '4' %}{% set coherence = message['coherence']|string or '4' %}{% set complexity = message['complexity']|string or '4' %}{% set verbosity = message['verbosity']|string or '4' %}{% set quality = message['quality']|string or '4' %}{% set toxicity = message['toxicity']|string or '0' %}{% set humor = message['humor']|string or '0' %}{% set creativity = message['creativity']|string or '0' %}{{ '[INST] ' + content + ' [ATTR] helpfulness: ' + helpfulness + ' correctness: ' + correctness + ' coherence: ' + coherence + ' complexity: ' + complexity + ' verbosity: ' + verbosity + ' quality: ' + quality + ' toxicity: ' + toxicity + ' humor: ' + humor + ' creativity: ' + creativity + ' [/ATTR] [/INST]' }}{% elif message['role'] == 'assistant' %}{{ content + eos_token }}{% endif %}{% endfor %}",
	"clean_up_tokenization_spaces": false,
	"eos_token": "</s>",
	"legacy": true,
	"model_max_length": 1000000000000000019884624838656,
	"pad_token": null,
	"sp_model_kwargs": {},
	"spaces_between_special_tokens": false,
	"tokenizer_class": "LlamaTokenizer",
	"unk_token": "<unk>",
	"use_default_system_prompt": false
}

# model-import-123456789012 のバケット名を適宜変更する
aws s3 sync karakuri-lm-8x7b-chat-v0.1/ s3://model-import-XXXXXXXXXXXX/karakuri-lm-8x7b-chat-v0.1

import boto3
from botocore.config import Config

config = Config(
    retries={
        'max_attempts': 10,
        'mode': 'standard'
    }
)

client = boto3.client("bedrock-runtime", region_name="us-east-1")

model_id = 'arn:aws:bedrock:us-east-1:123456789012:imported-model/XXXXXXXXXX'

prompt = "こんにちは！あなたの名前は何ですか？"

try:
    streaming_response = client.converse_stream(
        modelId=model_id,
        messages=[
            {
                "role": "user",
                "content": [{"text": prompt}],
            }
        ],
        inferenceConfig={"maxTokens": 512, "temperature": 1.0, "topP": 0.9},
    )

    for chunk in streaming_response["stream"]:
        if "contentBlockDelta" in chunk:
            text = chunk["contentBlockDelta"]["delta"]["text"]
            print(text, end="")
except Exception as e:
    print(e)
    print(e.__repr__())

(custom-model-import-py3.12) takakuni@ app % python converse.py
 こんにちは！私はKARAKURI LMといいます。カラクリ株式会社によって開発された大規模言語モデルです。どんな質問でもお答えできるので、みなさんのお役に立てれば幸いです。%